{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/mink/notebooks/CameraTraps')  # append this repo to PYTHONPATH\n",
    "sys.path.append('/home/mink/lib/ai4eutils')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import Counter, defaultdict\n",
    "from random import sample\n",
    "from shutil import copyfile\n",
    "from multiprocessing.pool import ThreadPool\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "import path_utils  # ai4eutils\n",
    "\n",
    "from data_management.megadb.schema import sequences_schema_check\n",
    "from data_management.megadb.converters.cct_to_megadb import process_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# wpz_wolverine_labeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'wpz_wolverine_labeled'\n",
    "\n",
    "container_root = '/mink_disk_0/camtraps/wpz/'  \n",
    "path_prefix = 'wolverine_survey/images/Wolverine Monitoring Photos/'\n",
    "\n",
    "path_to_output = f'/home/mink/camtraps/data/megadb_jsons/{dataset_name}.json' \n",
    "path_to_output_temp = f'/home/mink/camtraps/data/megadb_jsons/{dataset_name}_temp.json' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 0 - Add an entry to the `datasets` table\n",
    "\n",
    "Done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Prepare the `sequence` objects to insert into the database\n",
    "\n",
    "Species in a CSV. State info is in the file names.\n",
    "\n",
    "Sometimes the suffix of filename is lower-cased in the CSV.\n",
    "\n",
    "Emtpy is denoted \"None\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = os.path.join(container_root, 'wolverine_survey/images/All4StatesIDs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "377756"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ImageID</th>\n",
       "      <th>FileName</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>306160</th>\n",
       "      <td>49791</td>\n",
       "      <td>WY_Image_050141.jpg</td>\n",
       "      <td>Magpie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90866</th>\n",
       "      <td>94868</td>\n",
       "      <td>ID_Image_090868.JPG</td>\n",
       "      <td>Human</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ImageID             FileName   Label\n",
       "306160    49791  WY_Image_050141.jpg  Magpie\n",
       "90866     94868  ID_Image_090868.JPG   Human"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)\n",
    "df.sample(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Human', 'Wolverine', 'None', 'American Marten', 'Red Fox',\n",
       "       'Coyote', 'Magpie', 'Hawk', 'Snowshoe Hare', \"Clark's Nutcracker\",\n",
       "       'Wolf', 'Mountain Lion', 'Gray Jay', 'Mule Deer',\n",
       "       'Rocky Mountain Elk', \"Steller's Jay\", 'Unk Mammal',\n",
       "       'Golden eagle', 'Unk Bird', 'Domestic dog', 'Fisher', 'unknown',\n",
       "       'White-tailed Deer', 'Moose', 'Red Squirrel', 'Common Raven',\n",
       "       'Chickadee', 'Woodpecker', 'Flying Squirrel', 'Goshawk', 'Bobcat',\n",
       "       'skunk', 'Grizzly Bear', 'Black Bear', 'Blue Grouse',\n",
       "       'Golden-mantled GS', 'Timelapse Photo', 'Canada Lynx',\n",
       "       'Long-tailed Weasel', 'Mouse', 'Weasel', 'Researcher', '11amTest',\n",
       "       'Mountain goat', 'Cascade red fox', 'Douglas squirrel',\n",
       "       'Misc Squirrel', 'American Black Bear', 'grouse',\n",
       "       'Northern Flying Squirrel', 'American Badger', 'Misc Bird',\n",
       "       'Misc Small Rodent', 'Misc Chipmunk', 'Badger', 'Time Lapse Photo',\n",
       "       'Pacific Marten', 'Unknown', 'Raven', 'Elk', 'Golden Eagle',\n",
       "       'Northern Goshawk', 'Owl', 'Mustela spp.', 'Dusky Grouse',\n",
       "       'Porcupine', 'Striped Skunk', 'Downy Woodpecker'], dtype=object)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "377756"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "377756"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folder = os.path.join(container_root, path_prefix)\n",
    "\n",
    "paths = path_utils.recursive_file_list(folder)\n",
    "len(paths)\n",
    "paths = sorted([p.split(folder)[1] for p in paths if path_utils.is_image_file(p) and not os.path.basename(p).startswith('.')])\n",
    "len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "377756"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn_to_path = {}  # no extension to full file name\n",
    "for p in paths:\n",
    "    fn_to_path[p.split('.')[0]] = p\n",
    "len(fn_to_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WY_Image_121637.JPG'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[-100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WY_Image_050141.JPG'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn_to_path['WY_Image_050141']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "377756it [00:42, 8810.92it/s]\n"
     ]
    }
   ],
   "source": [
    "locations = set()\n",
    "sequences = []\n",
    "\n",
    "for i_row, row in tqdm(df.iterrows()):\n",
    "    \n",
    "    df_fn = row['FileName']\n",
    "    fn = fn_to_path[df_fn.split('.')[0]]\n",
    "    \n",
    "    location = df_fn.split('_')[0]\n",
    "    locations.add(location)\n",
    "    \n",
    "    # '11amTest' and 'Time Lapse Photo' usually empty but leaving it\n",
    "    clss = row['Label'].lower()\n",
    "    if clss == 'none':\n",
    "        clss = 'empty'\n",
    "    elif clss == 'researcher':\n",
    "        clss = 'human'\n",
    "        \n",
    "    sequences.append({\n",
    "        'dataset': dataset_name,\n",
    "        'seq_id': f'dummy_{len(sequences)}',\n",
    "        'images': [\n",
    "            {\n",
    "                'file': fn,\n",
    "                'frame_num': 1, # only one image, but easier for ingesting the annotations\n",
    "                'image_id': row['ImageID'] # extra field\n",
    "            }\n",
    "        ],\n",
    "        'location': location,\n",
    "        'class': [clss]\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69568"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_empty = sum([1 for seq in sequences if seq['class'][0] == 'empty'])\n",
    "num_empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18416120458708796"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_empty / 377756"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "{'ID', 'MT', 'WA', 'WY'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(locations)\n",
    "locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "377756"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'wpz_wolverine_labeled',\n",
       "  'seq_id': 'dummy_20000',\n",
       "  'images': [{'file': 'ID_Image_020002.JPG',\n",
       "    'frame_num': 1,\n",
       "    'image_id': 21733}],\n",
       "  'location': 'ID',\n",
       "  'class': ['red fox']},\n",
       " {'dataset': 'wpz_wolverine_labeled',\n",
       "  'seq_id': 'dummy_20001',\n",
       "  'images': [{'file': 'ID_Image_020003.JPG',\n",
       "    'frame_num': 1,\n",
       "    'image_id': 21734}],\n",
       "  'location': 'ID',\n",
       "  'class': ['red fox']},\n",
       " {'dataset': 'wpz_wolverine_labeled',\n",
       "  'seq_id': 'dummy_20002',\n",
       "  'images': [{'file': 'ID_Image_020004.JPG',\n",
       "    'frame_num': 1,\n",
       "    'image_id': 21735}],\n",
       "  'location': 'ID',\n",
       "  'class': ['red fox']}]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sequences)\n",
    "sequences[20000:20003]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Pass the schema check\n",
    "\n",
    "Once your metadata are in the MegaDB format for `sequence` items, we check that they conform to the format's schema.\n",
    "\n",
    "If the format conforms, the following messages will be printed:\n",
    "\n",
    "```\n",
    "Verified that the sequence items meet requirements not captured by the schema.\n",
    "Verified that the sequence items conform to the schema.\n",
    "```\n",
    "\n",
    "For large datasets, the second step will take some time (~ a minute). \n",
    "\n",
    "Otherwise there will be an error message describing what's wrong. Please fix the issues until all checks are passed. You might need to write some snippets of code to loop through the `sequence` items to understand which entries have problems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n",
      "CPU times: user 44.8 s, sys: 33.4 ms, total: 44.9 s\n",
      "Wall time: 44.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path_to_output_temp, 'w', encoding='utf-8') as f:\n",
    "    json.dump(sequences, f, indent=1, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2b - sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "125920"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_to_include = []\n",
    "\n",
    "batches_folder = 0\n",
    "for i in range(0, len(sequences), 10):  # 10 sequential images, take 1/4\n",
    "    if batches_folder % 3 == 0:\n",
    "        batch = sequences[i: i + 10]\n",
    "        sequences_to_include.extend(batch)\n",
    "    batches_folder += 1\n",
    "\n",
    "len(sequences_to_include)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102695"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_to_include = [seq for seq in sequences_to_include if not seq['class'][0] == 'empty']\n",
    "len(sequences_to_include)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences_to_include[100000: 100000 + 40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'wpz_wolverine_labeled',\n",
       "  'seq_id': 'dummy_228334',\n",
       "  'images': [{'file': 'WA_Image_025671.JPG',\n",
       "    'frame_num': 1,\n",
       "    'image_id': 32583}],\n",
       "  'location': 'WA',\n",
       "  'class': ['common raven']}]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(sequences_to_include, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2c - copy images to flat folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def copy_file(src_path, dst_path):\n",
    "    return copyfile(src_path, dst_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 102695/102695 [00:00<00:00, 115059.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 744 ms, sys: 169 ms, total: 913 ms\n",
      "Wall time: 906 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "path_pairs = []\n",
    "for seq in tqdm(sequences_to_include):\n",
    "    seq_id = seq['seq_id']\n",
    "    \n",
    "    im = seq['images'][0]\n",
    "    frame = im['frame_num']\n",
    "    im_basename = im['file']  # inc extension\n",
    "    \n",
    "    src_path = os.path.join(container_root, path_prefix, im['file'])\n",
    "    assert os.path.exists(src_path), src_path\n",
    "\n",
    "    dst_path = os.path.join('/mink_disk_0/camtraps/imerit12c', \n",
    "                            f'{dataset_name}.seq{seq_id}.frame{frame}.file{im_basename}')\n",
    "    path_pairs.append((src_path, dst_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102695"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "('/mink_disk_0/camtraps/wpz/wolverine_survey/images/Wolverine Monitoring Photos/ID_Image_011644.JPG',\n",
       " '/mink_disk_0/camtraps/imerit12c/wpz_wolverine_labeled.seqdummy_11642.frame1.fileID_Image_011644.JPG')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(path_pairs)  # non-empty images out of total of 19221 (38%)\n",
    "path_pairs[3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 39.5 s, sys: 1min 19s, total: 1min 59s\n",
      "Wall time: 4min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "with ThreadPool(8) as pool:\n",
    "    dst_paths = pool.starmap(copy_file, path_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102695"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dst_paths)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cameratraps] *",
   "language": "python",
   "name": "conda-env-cameratraps-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
